{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Compare performance of different CNN architectures"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We compare **performance**:\n",
    "- of 4 different CNN **models** (net + weights):\n",
    "  - GoogleNet;\n",
    "  - AlexNet;\n",
    "  - SqueezeNet 1.0;\n",
    "  - SqueezeNet 1.1;\n",
    "\n",
    "- when using 4 different **libraries**:\n",
    "\n",
    "  - [CPU] clblast_mali_overlay 0.2.18;\n",
    "  - [GPU] clBLAS 2.4;\n",
    "  - [GPU] CLBlast dev (> 0.8.0);\n",
    "  - [GPU] CLBlast dev (> 0.8.0) with Mali-optimised overlay."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Includes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Standard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import sys\n",
    "import json"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Scientific"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import IPython as ip\n",
    "import numpy as np\n",
    "import scipy as sp\n",
    "import pandas as pd\n",
    "import matplotlib as mp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "print 'IPython version: %s' % ip.__version__\n",
    "print 'NumPy version: %s' % np.__version__\n",
    "print 'SciPy version: %s' % sp.__version__\n",
    "print 'Pandas version: %s' % pd.__version__\n",
    "print 'Matplotlib version: %s' % mp.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "from matplotlib import cm\n",
    "%matplotlib inline\n",
    "# import scipy.stats as st"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Collective Knowledge"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import ck.kernel as ck\n",
    "print 'CK version: %s' % ck.__version__"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Access experimental results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def search_experimental_points_by_tags(tags):\n",
    "    r=ck.access({'action':'search', 'module_uoa':'experiment', 'tags':tags})\n",
    "    if r['return']>0:\n",
    "        print (\"Error: %s\" % r['error'])\n",
    "        exit(1)\n",
    "    # FIXME: For now, assume a single entry per the given tags.\n",
    "    r=ck.access({'action':'list_points', 'module_uoa':'experiment', 'data_uoa': r['lst'][0]['data_uoa']})\n",
    "    if r['return']>0:\n",
    "        print (\"Error: %s\" % r['error'])\n",
    "        exit(1)\n",
    "    \n",
    "    results = {}\n",
    "    for point in r['points']:\n",
    "        with open(os.path.join(r['path'], 'ckp-%s.0001.json' % point)) as point_file:\n",
    "            point_data_raw = json.load(point_file)\n",
    "            point_data_dict = {}\n",
    "            time_fw_ms = [\n",
    "                characteristics['run']['time_fw_ms'] \n",
    "                for characteristics in point_data_raw['characteristics_list']\n",
    "                if characteristics['run']['run_success'] == 'yes'\n",
    "            ]\n",
    "            batch_size = point_data_raw['choices']['env']['CK_CAFFE_BATCH_SIZE']\n",
    "            results[batch_size] = time_fw_ms\n",
    "    return results\n",
    "\n",
    "def get_min_time_per_image(results):\n",
    "    df = pd.DataFrame(data=results)\n",
    "    df.index.name = 'repetition'\n",
    "    df.columns.name = 'batch size'\n",
    "    return (df.describe().ix['min'] / range(1,len(results)+1)).min()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### AlexNet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "alexnet_clblas_tags = 'time,caffemodel,alexnet,clblas'\n",
    "alexnet_clblas_results = search_experimental_points_by_tags(alexnet_clblas_tags)\n",
    "alexnet_clblas_min_time_per_image = get_min_time_per_image(alexnet_clblas_results)\n",
    "alexnet_clblas_min_time_per_image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "alexnet_clblast_development_tags = 'time,caffemodel,alexnet,clblast,vdevelopment'\n",
    "alexnet_clblast_development_results = search_experimental_points_by_tags(alexnet_clblast_development_tags)\n",
    "alexnet_clblast_development_min_time_per_image = get_min_time_per_image(alexnet_clblast_development_results)\n",
    "alexnet_clblast_development_min_time_per_image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "alexnet_clblast_mali_overlay_tags = 'time,caffemodel,alexnet,clblast,vmali-overlay'\n",
    "alexnet_clblast_mali_overlay_results = search_experimental_points_by_tags(alexnet_clblast_mali_overlay_tags)\n",
    "alexnet_clblast_mali_overlay_min_time_per_image = get_min_time_per_image(alexnet_clblast_mali_overlay_results)\n",
    "alexnet_clblast_mali_overlay_min_time_per_image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "alexnet_openblas_tags = 'time,caffemodel,alexnet,openblas'\n",
    "alexnet_openblas_results = search_experimental_points_by_tags(alexnet_openblas_tags)\n",
    "alexnet_openblas_min_time_per_image = get_min_time_per_image(alexnet_openblas_results)\n",
    "alexnet_openblas_min_time_per_image"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### SqueezeNet 1.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "squeezenet_1_0_clblas_tags = 'time,caffemodel,squeezenet-1.0,clblas'\n",
    "squeezenet_1_0_clblas_results = search_experimental_points_by_tags(squeezenet_1_0_clblas_tags)\n",
    "squeezenet_1_0_clblas_min_time_per_image = get_min_time_per_image(squeezenet_1_0_clblas_results)\n",
    "squeezenet_1_0_clblas_min_time_per_image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "squeezenet_1_0_clblast_development_tags = 'time,caffemodel,squeezenet-1.0,clblast,vdevelopment'\n",
    "squeezenet_1_0_clblast_development_results = search_experimental_points_by_tags(squeezenet_1_0_clblast_development_tags)\n",
    "squeezenet_1_0_clblast_development_min_time_per_image = get_min_time_per_image(squeezenet_1_0_clblast_development_results)\n",
    "squeezenet_1_0_clblast_development_min_time_per_image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "squeezenet_1_0_clblast_mali_overlay_tags = 'time,caffemodel,squeezenet-1.0,clblast,vmali-overlay'\n",
    "squeezenet_1_0_clblast_mali_overlay_results = search_experimental_points_by_tags(squeezenet_1_0_clblast_mali_overlay_tags)\n",
    "squeezenet_1_0_clblast_mali_overlay_min_time_per_image = get_min_time_per_image(squeezenet_1_0_clblast_mali_overlay_results)\n",
    "squeezenet_1_0_clblast_mali_overlay_min_time_per_image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "squeezenet_1_0_openblas_tags = 'time,caffemodel,squeezenet-1.0,openblas'\n",
    "squeezenet_1_0_openblas_results = search_experimental_points_by_tags(squeezenet_1_0_openblas_tags)\n",
    "squeezenet_1_0_openblas_min_time_per_image = get_min_time_per_image(squeezenet_1_0_openblas_results)\n",
    "squeezenet_1_0_openblas_min_time_per_image"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### SqueezeNet 1.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "squeezenet_1_1_clblas_tags = 'time,caffemodel,squeezenet-1.1,clblas'\n",
    "squeezenet_1_1_clblas_results = search_experimental_points_by_tags(squeezenet_1_1_clblas_tags)\n",
    "squeezenet_1_1_clblas_min_time_per_image = get_min_time_per_image(squeezenet_1_1_clblas_results)\n",
    "squeezenet_1_1_clblas_min_time_per_image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "squeezenet_1_1_clblast_development_tags = 'time,caffemodel,squeezenet-1.1,clblast,vdevelopment'\n",
    "squeezenet_1_1_clblast_development_results = search_experimental_points_by_tags(squeezenet_1_1_clblast_development_tags)\n",
    "squeezenet_1_1_clblast_development_min_time_per_image = get_min_time_per_image(squeezenet_1_1_clblast_development_results)\n",
    "squeezenet_1_1_clblast_development_min_time_per_image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "squeezenet_1_1_clblast_mali_overlay_tags = 'time,caffemodel,squeezenet-1.1,clblast,vmali-overlay'\n",
    "squeezenet_1_1_clblast_mali_overlay_results = search_experimental_points_by_tags(squeezenet_1_1_clblast_mali_overlay_tags)\n",
    "squeezenet_1_1_clblast_mali_overlay_min_time_per_image = get_min_time_per_image(squeezenet_1_1_clblast_mali_overlay_results)\n",
    "squeezenet_1_1_clblast_mali_overlay_min_time_per_image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "squeezenet_1_1_openblas_tags = 'time,caffemodel,squeezenet-1.1,openblas'\n",
    "squeezenet_1_1_openblas_results = search_experimental_points_by_tags(squeezenet_1_1_openblas_tags)\n",
    "squeezenet_1_1_openblas_min_time_per_image = get_min_time_per_image(squeezenet_1_1_openblas_results)\n",
    "squeezenet_1_1_openblas_min_time_per_image"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### GoogleNet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "googlenet_clblas_tags = 'time,caffemodel,googlenet,clblas'\n",
    "googlenet_clblas_results = search_experimental_points_by_tags(googlenet_clblas_tags)\n",
    "googlenet_clblas_min_time_per_image = get_min_time_per_image(googlenet_clblas_results)\n",
    "googlenet_clblas_min_time_per_image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "googlenet_clblast_development_tags = 'time,caffemodel,googlenet,clblast,vdevelopment'\n",
    "googlenet_clblast_development_results = search_experimental_points_by_tags(googlenet_clblast_development_tags)\n",
    "googlenet_clblast_development_min_time_per_image = get_min_time_per_image(googlenet_clblast_development_results)\n",
    "googlenet_clblast_development_min_time_per_image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "googlenet_clblast_mali_overlay_tags = 'time,caffemodel,googlenet,clblast,vmali-overlay'\n",
    "googlenet_clblast_mali_overlay_results = search_experimental_points_by_tags(googlenet_clblast_mali_overlay_tags)\n",
    "googlenet_clblast_mali_overlay_min_time_per_image = get_min_time_per_image(googlenet_clblast_mali_overlay_results)\n",
    "googlenet_clblast_mali_overlay_min_time_per_image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "googlenet_openblas_tags = 'time,caffemodel,googlenet,openblas'\n",
    "googlenet_openblas_results = search_experimental_points_by_tags(googlenet_openblas_tags)\n",
    "googlenet_openblas_min_time_per_image = get_min_time_per_image(googlenet_openblas_results)\n",
    "googlenet_openblas_min_time_per_image"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data frame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "data = {\n",
    "    'AlexNet' : {\n",
    "        'OpenBLAS: v0.2.18'     : alexnet_openblas_min_time_per_image,\n",
    "        'clBLAS: v2.4'          : alexnet_clblas_min_time_per_image,\n",
    "        'CLBlast: development'  : alexnet_clblast_development_min_time_per_image,\n",
    "        'CLBlast: Mali overlay' : alexnet_clblast_mali_overlay_min_time_per_image\n",
    "    },\n",
    "#     'GoogleNet' : {\n",
    "#         'OpenBLAS: v0.2.18'     : googlenet_openblas_min_time_per_image,\n",
    "#         'clBLAS: v2.4'          : googlenet_clblas_min_time_per_image,\n",
    "#         'CLBlast: development'  : googlenet_clblast_development_min_time_per_image,\n",
    "#         'CLBlast: Mali overlay' : googlenet_clblast_mali_overlay_min_time_per_image\n",
    "#     },\n",
    "    'SqueezeNet 1.0' : {\n",
    "        'OpenBLAS: v0.2.18'     : squeezenet_1_0_openblas_min_time_per_image,\n",
    "        'clBLAS: v2.4'          : squeezenet_1_0_clblas_min_time_per_image,\n",
    "        'CLBlast: development'  : squeezenet_1_0_clblast_development_min_time_per_image,\n",
    "        'CLBlast: Mali overlay' : squeezenet_1_0_clblast_mali_overlay_min_time_per_image\n",
    "    },\n",
    "    'SqueezeNet 1.1' : {\n",
    "        'OpenBLAS: v0.2.18'     : squeezenet_1_1_openblas_min_time_per_image,\n",
    "        'clBLAS: v2.4'          : squeezenet_1_1_clblas_min_time_per_image,\n",
    "        'CLBlast: development'  : squeezenet_1_1_clblast_development_min_time_per_image,\n",
    "        'CLBlast: Mali overlay' : squeezenet_1_1_clblast_mali_overlay_min_time_per_image\n",
    "    },\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df = pd.DataFrame(data)\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df.T \\\n",
    "    .plot(title='Execution time per image (ms)',\n",
    "          kind='bar', rot=0, ylim=[0,1100], figsize=[12, 6], grid=True, legend=True, colormap=cm.autumn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df \\\n",
    "    .plot(title='Execution time per image (ms)',\n",
    "          kind='bar', rot=0, ylim=[0,1100], figsize=[12, 6], grid=True, legend=True, colormap=cm.autumn)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
